Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Merge pull request #466 from knu/website_agent-response_drop

Give access to response status and header values in WebsiteAgent.

Closes #445.

Akinori MUSHA 10 years ago
parent
5a8206401a 2b493f548b
commit
10ecc8b7d2
7 changed files with 201 additions and 76 deletions
Show Diff Stats
  1. 7 39
      app/concerns/liquid_droppable.rb
  2. 78 12
      app/concerns/liquid_interpolatable.rb
  3. 6 5
      app/models/agents/event_formatting_agent.rb
  4. 36 13
      app/models/agents/website_agent.rb
  5. 13 6
      app/models/event.rb
  6. 32 1
      spec/models/agents/website_agent_spec.rb
  7. 29 0
      spec/support/shared_examples/liquid_interpolatable.rb

+ 7 - 39
app/concerns/liquid_droppable.rb
View File 

@@ -1,48 +1,16 @@
1 
+# Include this mix-in to make a class droppable to Liquid, and adjust
2 
+# its behavior in Liquid by implementing its dedicated Drop class
3 
+# named with a "Drop" suffix.
1 4 
 module LiquidDroppable
2 5 
   extend ActiveSupport::Concern
3 6
4 
-  # In subclasses of this base class, "locals" take precedence over
5 
-  # methods.
6 7 
   class Drop < Liquid::Drop
7 
-    class << self
8 
-      def inherited(subclass)
9 
-        class << subclass
10 
-          attr_reader :drop_methods
11 
-
12 
-          # Make all public methods private so that #before_method
13 
-          # catches everything.
14 
-          def drop_methods!
15 
-            return if @drop_methods
16 
-
17 
-            @drop_methods = Set.new
18 
-
19 
-            (public_instance_methods - Drop.public_instance_methods).each { |name|
20 
-              @drop_methods << name.to_s
21 
-              private name
22 
-            }
23 
-          end
24 
-        end
25 
-      end
26 
-    end
27 
-
28 
-    def initialize(object, locals = nil)
29 
-      self.class.drop_methods!
30 
-
8 
+    def initialize(object)
31 9 
       @object = object
32 
-      @locals = locals || {}
33 
-    end
34 
-
35 
-    def before_method(name)
36 
-      if @locals.include?(name)
37 
-        @locals[name]
38 
-      elsif self.class.drop_methods.include?(name)
39 
-        __send__(name)
40 
-      end
41 10 
     end
42 11
43 12 
     def each
44 
-      return to_enum(__method__) unless block_given?
45 
-      self.class.drop_methods.each { |name|
13 
+      (public_instance_methods - Drop.public_instance_methods).each { |name|
46 14 
         yield [name, __send__(name)]
47 15 
       }
48 16 
     end
 
@@ -52,7 +20,7 @@ module LiquidDroppable
52 20 
     const_set :Drop, Kernel.const_set("#{name}Drop", Class.new(Drop))
53 21 
   end
54 22
55 
-  def to_liquid(*args)
56 
-    self.class::Drop.new(self, *args)
23 
+  def to_liquid
24 
+    self.class::Drop.new(self)
57 25 
   end
58 26 
 end

+ 78 - 12
app/concerns/liquid_interpolatable.rb
View File 

@@ -18,28 +18,94 @@ module LiquidInterpolatable
18 18 
     false
19 19 
   end
20 20
21 
-  def interpolate_options(options, event = {})
22 
-    case options
21 
+  # Return the current interpolation context.  Use this in your Agent
22 
+  # class to manipulate interpolation context for user.
23 
+  #
24 
+  # For example, to provide local variables:
25 
+  #
26 
+  #     # Create a new scope to define variables in:
27 
+  #     interpolation_context.stack {
28 
+  #       interpolation_context['_something_'] = 42
29 
+  #       # And user can say "{{_something_}}" in their options.
30 
+  #       value = interpolated['some_key']
31 
+  #     }
32 
+  #
33 
+  def interpolation_context
34 
+    @interpolation_context ||= Context.new(self)
35 
+  end
36 
+
37 
+  # Take the given object as "self" in the current interpolation
38 
+  # context while running a given block.
39 
+  #
40 
+  # The most typical use case for this is to evaluate options for each
41 
+  # received event like this:
42 
+  #
43 
+  #     def receive(incoming_events)
44 
+  #       incoming_events.each do |event|
45 
+  #         interpolate_with(event) do
46 
+  #           # Handle each event based on "interpolated" options.
47 
+  #         end
48 
+  #       end
49 
+  #     end
50 
+  def interpolate_with(self_object)
51 
+    case self_object
52 
+    when nil
53 
+      yield
54 
+    else
55 
+      context = interpolation_context
56 
+      begin
57 
+        context.environments.unshift(self_object.to_liquid)
58 
+        yield
59 
+      ensure
60 
+        context.environments.shift
61 
+      end
62 
+    end
63 
+  end
64 
+
65 
+  def interpolate_options(options, self_object = nil)
66 
+    interpolate_with(self_object) do
67 
+      case options
23 68 
       when String
24 
-        interpolate_string(options, event)
69 
+        interpolate_string(options)
25 70 
       when ActiveSupport::HashWithIndifferentAccess, Hash
26 
-        options.inject(ActiveSupport::HashWithIndifferentAccess.new) { |memo, (key, value)| memo[key] = interpolate_options(value, event); memo }
71 
+        options.each_with_object(ActiveSupport::HashWithIndifferentAccess.new) { |(key, value), memo|
72 
+          memo[key] = interpolate_options(value)
73 
+        }
27 74 
       when Array
28 
-        options.map { |value| interpolate_options(value, event) }
75 
+        options.map { |value| interpolate_options(value) }
29 76 
       else
30 77 
         options
78 
+      end
79 
+    end
80 
+  end
81 
+
82 
+  def interpolated(self_object = nil)
83 
+    interpolate_with(self_object) do
84 
+      (@interpolated_cache ||= {})[[options, interpolation_context]] ||=
85 
+        interpolate_options(options)
31 86 
     end
32 87 
   end
33 88
34 
-  def interpolated(event = {})
35 
-    key = [options, event]
36 
-    @interpolated_cache ||= {}
37 
-    @interpolated_cache[key] ||= interpolate_options(options, event)
38 
-    @interpolated_cache[key]
89 
+  def interpolate_string(string, self_object = nil)
90 
+    interpolate_with(self_object) do
91 
+      Liquid::Template.parse(string).render!(interpolation_context)
92 
+    end
39 93 
   end
40 94
41 
-  def interpolate_string(string, event)
42 
-    Liquid::Template.parse(string).render!(event.to_liquid, registers: {agent: self})
95 
+  class Context < Liquid::Context
96 
+    def initialize(agent)
97 
+      super({}, {}, { agent: agent }, true)
98 
+    end
99 
+
100 
+    def hash
101 
+      [@environments, @scopes, @registers].hash
102 
+    end
103 
+
104 
+    def eql?(other)
105 
+      other.environments == @environments &&
106 
+        other.scopes == @scopes &&
107 
+        other.registers == @registers
108 
+    end
43 109 
   end
44 110
45 111 
   require 'uri'

+ 6 - 5
app/models/agents/event_formatting_agent.rb
View File 

@@ -120,11 +120,12 @@ module Agents
120 120
121 121 
     def receive(incoming_events)
122 122 
       incoming_events.each do |event|
123 
-        payload = perform_matching(event.payload)
124 
-        opts = interpolated(event.to_liquid(payload))
125 
-        formatted_event = opts['mode'].to_s == "merge" ? event.payload.dup : {}
126 
-        formatted_event.merge! opts['instructions']
127 
-        create_event :payload => formatted_event
123 
+        interpolate_with(event) do
124 
+          interpolation_context.merge(perform_matching(event.payload))
125 
+          formatted_event = interpolated['mode'].to_s == "merge" ? event.payload.dup : {}
126 
+          formatted_event.merge! interpolated['instructions']
127 
+          create_event :payload => formatted_event
128 
+        end
128 129 
       end
129 130 
     end
130 131

+ 36 - 13
app/models/agents/website_agent.rb
View File 

@@ -75,6 +75,14 @@ module Agents
75 75 
       The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
76 76
77 77 
       The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
78 
+
79 
+      In Liquid templating, the following variable is available:
80 
+
81 
+      * `_response_`: A response object with the following keys:
82 
+
83 
+          * `status`: HTTP status as integer. (Almost always 200)
84 
+
85 
+          * `headers`: Reponse headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insentitive to cases and -/_.
78 86 
     MD
79 87
80 88 
     event_description do
 
@@ -149,7 +157,10 @@ module Agents
149 157 
       Array(in_url).each do |url|
150 158 
         log "Fetching #{url}"
151 159 
         response = faraday.get(url)
152 
-        if response.success?
160 
+        raise "Failed: #{response.inspect}" unless response.success?
161 
+
162 
+        interpolation_context.stack {
163 
+          interpolation_context['_response_'] = ResponseDrop.new(response)
153 164 
           body = response.body
154 165 
           if (encoding = interpolated['force_encoding']).present?
155 166 
             body = body.encode(Encoding::UTF_8, encoding)
 
@@ -195,9 +206,7 @@ module Agents
195 206 
               create_event :payload => result
196 207 
             end
197 208 
           end
198 
-        else
199 
-          raise "Failed: #{response.inspect}"
200 
-        end
209 
+        }
201 210 
       end
202 211 
     rescue => e
203 212 
       error e.message
 
@@ -205,16 +214,11 @@ module Agents
205 214
206 215 
     def receive(incoming_events)
207 216 
       incoming_events.each do |event|
208 
-        Thread.current[:current_event] = event
209 
-        url_to_scrape = event.payload['url']
210 
-        check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
217 
+        interpolate_with(event) do
218 
+          url_to_scrape = event.payload['url']
219 
+          check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
220 
+        end
211 221 
       end
212 
-    ensure
213 
-      Thread.current[:current_event] = nil
214 
-    end
215 
-
216 
-    def interpolated(event = Thread.current[:current_event])
217 
-      super
218 222 
     end
219 223
220 224 
     private
 
@@ -349,5 +353,24 @@ module Agents
349 353 
     rescue
350 354 
       false
351 355 
     end
356 
+
357 
+    # Wraps Faraday::Response
358 
+    class ResponseDrop < LiquidDroppable::Drop
359 
+      def headers
360 
+        HeaderDrop.new(@object.headers)
361 
+      end
362 
+
363 
+      # Integer value of HTTP status
364 
+      def status
365 
+        @object.status
366 
+      end
367 
+    end
368 
+
369 
+    # Wraps Faraday::Utilsa::Headers
370 
+    class HeaderDrop < LiquidDroppable::Drop
371 
+      def before_method(name)
372 
+        @object[name.tr('_', '-')]
373 
+      end
374 
+    end
352 375 
   end
353 376 
 end

+ 13 - 6
app/models/event.rb
View File 

@@ -48,21 +48,28 @@ class Event < ActiveRecord::Base
48 48 
 end
49 49
50 50 
 class EventDrop
51 
-  def initialize(object, locals = nil)
52 
-    locals = object.payload.merge(locals || {})
51 
+  def initialize(object)
52 
+    @payload = object.payload
53 53 
     super
54 54 
   end
55 55
56 
+  def before_method(key)
57 
+    @payload[key]
58 
+  end
59 
+
56 60 
   def each(&block)
57 
-    return to_enum(__method__) unless block
58 
-    @locals.each(&block)
61 
+    @payload.each(&block)
59 62 
   end
60 63
61 64 
   def agent
62 
-    @object.agent
65 
+    @payload.fetch(__method__) {
66 
+      @object.agent
67 
+    }
63 68 
   end
64 69
65 70 
   def created_at
66 
-    @object.created_at
71 
+    @payload.fetch(__method__) {
72 
+      @object.created_at
73 
+    }
67 74 
   end
68 75 
 end

+ 32 - 1
spec/models/agents/website_agent_spec.rb
View File 

@@ -3,7 +3,11 @@ require 'spec_helper'
3 3 
 describe Agents::WebsiteAgent do
4 4 
   describe "checking without basic auth" do
5 5 
     before do
6 
-      stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
6 
+      stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")),
7 
+                                           status: 200,
8 
+                                           headers: {
9 
+                                             'X-Status-Message' => 'OK'
10 
+                                           })
7 11 
       @valid_options = {
8 12 
         'name' => "XKCD",
9 13 
         'expected_update_period_in_days' => "2",
 
@@ -305,6 +309,17 @@ describe Agents::WebsiteAgent do
305 309 
         event.payload['slogan'].should == "A webcomic of romance, sarcasm, math, and language."
306 310 
       end
307 311
312 
+      it "should interpolate _response_" do
313 
+        @valid_options['extract']['response_info'] =
314 
+          @valid_options['extract']['url'].merge(
315 
+            'value' => '"{{ "The reponse was " | append:_response_.status | append:" " | append:_response_.headers.X-Status-Message | append:"." }}"'
316 
+          )
317 
+        @checker.options = @valid_options
318 
+        @checker.check
319 
+        event = Event.last
320 
+        event.payload['response_info'].should == 'The reponse was 200 OK.'
321 
+      end
322 
+
308 323 
       describe "JSON" do
309 324 
         it "works with paths" do
310 325 
           json = {
 
@@ -490,6 +505,22 @@ fire: hot
490 505 
           'to' => 'http://dynamic.xkcd.com/random/comic/',
491 506 
         }
492 507 
       end
508 
+
509 
+      it "should interpolate values from incoming event payload and _response_" do
510 
+        @event.payload['title'] = 'XKCD'
511 
+
512 
+        lambda {
513 
+          @valid_options['extract'] = {
514 
+            'response_info' => @valid_options['extract']['url'].merge(
515 
+              'value' => '{% capture sentence %}The reponse from {{title}} was {{_response_.status}} {{_response_.headers.X-Status-Message}}.{% endcapture %}{{sentence | to_xpath}}'
516 
+            )
517 
+          }
518 
+          @checker.options = @valid_options
519 
+          @checker.receive([@event])
520 
+        }.should change { Event.count }.by(1)
521 
+
522 
+        Event.last.payload['response_info'].should == 'The reponse from XKCD was 200 OK.'
523 
+      end
493 524 
     end
494 525 
   end
495 526

+ 29 - 0
spec/support/shared_examples/liquid_interpolatable.rb
View File 

@@ -54,6 +54,35 @@ shared_examples_for LiquidInterpolatable do
54 54 
       @checker.interpolate_string("{{variable}}", @event).should == "hello"
55 55 
       @checker.interpolate_string("{{variable}} you", @event).should == "hello you"
56 56 
     end
57 
+
58 
+    it "should use local variables while in a block" do
59 
+      @checker.options['locals'] = '{{_foo_}} {{_bar_}}'
60 
+
61 
+      @checker.interpolation_context.tap { |context|
62 
+        @checker.interpolated['locals'].should == ' '
63 
+
64 
+        context.stack {
65 
+          context['_foo_'] = 'This is'
66 
+          context['_bar_'] = 'great.'
67 
+
68 
+          @checker.interpolated['locals'].should == 'This is great.'
69 
+        }
70 
+
71 
+        @checker.interpolated['locals'].should == ' '
72 
+      }
73 
+    end
74 
+
75 
+    it "should use another self object while in a block" do
76 
+      @checker.options['properties'] = '{{_foo_}} {{_bar_}}'
77 
+
78 
+      @checker.interpolated['properties'].should == ' '
79 
+
80 
+      @checker.interpolate_with({ '_foo_' => 'That was', '_bar_' => 'nice.' }) {
81 
+        @checker.interpolated['properties'].should == 'That was nice.'
82 
+      }
83 
+
84 
+      @checker.interpolated['properties'].should == ' '
85 
+    end
57 86 
   end
58 87
59 88 
   describe "liquid tags" do